#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import pymysql
import requests
import random
import re
import time
from bs4 import BeautifulSoup


version = '0.0.0.2'
class TaoBao(object):
    """淘宝，对搜索结果进行数据提取

    """
    def __init__(self, k, s):
        self.keyword = k
        self.shop_nick = s
        self.sql = Mysql('taobao_test')
        self.search_url_hearder = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"}
        self.timeout = random.choice(range(80, 180))

    def func_req_search_view(self):
        """对关键字进行搜索，清洗数据

        :param keyword: 搜索关键字
        :return:
        """
        _search_url = 'https://s.taobao.com/search?'
        page = 0
        page_limit = 100 # 总页数,从返回的数据中提取
        page_size = 44 # 每页跨度,从返回的数据中提取
        default_url = [] # 解析失败时存储url
        while page < page_limit:
            _num = page * page_size
            _search_data = {'q':self.keyword, 'imgfile':'', 'js':'1', 'stats_click':'search_radio_all%3A1', 'ie':'utf8',
                           'sort':'sale-desc', 's':'%s' % _num}
            req = requests.get(url = _search_url, params = _search_data, headers = self.search_url_hearder,
                               timeout = self.timeout, stream = True)
            _bs = BeautifulSoup(req.content, "lxml")
            _scripts = _bs.head.find_all('script')
            time.sleep(1)
            # print(_scripts)
            for _string in _scripts[-1].stripped_strings:
                _r = re.search('{"pageName":(.*?)\n', _string)
                if _r and _num == 0:
                    data = json.loads(_r.group().rstrip(';\n'))
                    page_limit = 100 if int(data['mods']['sortbar']['data']['pager']['totalPage']) else int(data['mods']['sortbar']['data']['pager']['totalPage'])
                    page_size = int(data['mods']['sortbar']['data']['pager']['pageSize'])
                    _total_count = int(data['mods']['sortbar']['data']['pager']['totalCount']) # 总结果数
                    # print('页数：{}；总结果数：{}'.format(page_limit, _total_count))
                    self.func_result(data, page)
                elif _r:
                    data = json.loads(_r.group().rstrip(';\n'))
                    self.func_result(data, page)
                    time.sleep(1)
                else:
                    default_url.append(page)
                    # print('未返回数据，页码列表：{}'.format(default_url))
            page += 1
        else:
            # print('未返回数据，页码列表：{}'.format(default_url))
            print('-------结束查询-------')
            print('3分钟后自动退出！')
            time.sleep(180)
            exit()

    def func_result(self, items, page):
        """数据提取

        :param items: 存放数据的json
        :param page: 页码
        :return:
        """
        if items['mods']['itemlist']['status'] == 'show':
            _item_info = items['mods']['itemlist']['data']['auctions'] # 商品数据
            for item in _item_info:
                nick = item['nick'] # 店铺名
                if self.shop_nick == nick[0]:
                    # 处理店铺名
                    raw_title = item['raw_title'] # 商品标题
                    # user_id = int(item['user_id'])
                    view_price = item['view_price'] # 促销价的最小值，例：促销价 ¥ 79.90-99.00 取79.90
                    view_fee = item['view_fee'] # 运费
                    item_loc = item['item_loc'] # 发货地址
                    view_sales = item['view_sales'] # 收货人数量
                    comment_count = 0 if not item['comment_count'] else int(item['comment_count']) # 评论数
                    detail_url = item['detail_url'] # 详情页的地址
                    isTmall = item['shopcard']['isTmall'] # 是否天猫店
                    sql = ("""INSERT INTO taobao_test.shop_info_test(nick, raw_title, view_price, view_fee, item_loc,"""
                           """view_sales, comment_count, detail_url, isTmall) VALUES ("{nick}","{raw_title}","{view_price}","""
                           """"{view_fee}","{item_loc}","{view_sales}","{comment_count}","{detail_url}",{isTmall})""").format(nick = nick,
                                                                                                                              raw_title = raw_title,view_price = view_price,
                                                                                                                              view_fee = view_fee,item_loc = item_loc,
                                                                                                                              view_sales = view_sales,comment_count = comment_count,
                                                                                                                              detail_url = detail_url,isTmall = isTmall)
                    print('店铺名：%s，商品标题：%s，地址：%s，' % (nick,raw_title,detail_url))
                    print('继续查找...')
                    # print(sql)
                    self.Sql(sql)
        # else:
        #     print('end page: {}'.format(page))

    def Sql(self, s):
        """执行sql

        :param s: sql语句
        :return:
        """
        self.sql.cursor.execute(s)
        self.sql.conn.commit()

class Mysql(object):
    """连接数据库

    """
    global cursor, conn

    def __init__(self, db):
        self.connect(db)

    def connect(self, db):
        self.conn = pymysql.connect(host = '192.168.2.80', port = 3306, user = 'root', passwd = '', db = db, charset = 'utf8')
        self.cursor = self.conn.cursor(cursor = pymysql.cursors.DictCursor)

if __name__ == '__main__':
    print('版本号：' + version)
    keyword = input('请输入搜索商品关键字：')
    shop_nick = input('请输入店铺首字：')
    print('开始搜索，整个过程预计持续2分钟，请稍候！')
    t = TaoBao(keyword, shop_nick)
    t.func_req_search_view()